from scrapy.spiders import CrawlSpider
from scrapy.selector import Selector
from petspider.items import PetSpiderItem
from petspider.spiders.woff2xml import woff_change
from petspider.spiders.woff2xml import addr_change
from petspider.spiders.woff2xml import tag_change


# 功能：爬取南通市宠物店信息1
class petspider(CrawlSpider):
    name = 'petspider'
    allowed_domains = ['dianping.com']

    start_urls = ['http://www.dianping.com/search/keyword/94/95_%E5%AE%A0%E7%89%A9%E5%BA%97/p14']
    # start_urls = ['http://www.dianping.com/nantong/ch95/g25147p1']

    def parse(self, response):
        for i in range(1, 16):
            self.start_requests()
            item = PetSpiderItem()
            selector = Selector(response)
            city = '南通'
            shop = selector.xpath(
                '//*[@id="shop-all-list"]/ul/li[' + str(i) + ']/div[2]/div[1]/a/h4/text()').extract_first()
            url = selector.xpath('//*[@id="shop-all-list"]/ul/li[' + str(i) + ']/div[2]/div[1]/a/@href').extract_first()

            comment_path = '//*[@id="shop-all-list"]/ul/li[' + str(i) + ']/div[2]/div[2]/a[1]/b//text()'
            comments = response.xpath(comment_path).extract()
            temp_comment = woff_change(comments)
            temp_comment = temp_comment + '条评论'

            ac_path = '//*[@id="shop-all-list"]/ul/li[' + str(i) + ']/div[2]/div[2]/a[2]/b//text()'
            avg_price = response.xpath(ac_path).extract()
            temp_ac = woff_change(avg_price)
            temp_ac = '人均' + temp_ac

            addr_path_head = '//*[@id="shop-all-list"]/ul/li[' + str(i) + ']/div[2]/div[3]/a[2]/span//text()'
            addr_path_body = '//*[@id="shop-all-list"]/ul/li[' + str(i) + ']/div[2]/div[3]/span//text()'
            path_head = response.xpath(addr_path_head).extract()
            path_body = response.xpath(addr_path_body).extract()
            temp_addr = tag_change(path_head) + ' ' + addr_change(path_body)

            item['city'] = str(city)
            item['url'] = str(url)
            item['shop'] = str(shop)
            if temp_comment == '条评论':
                item['comments_count'] = str(str(0) + temp_comment)
            else:
                item['comments_count'] = str(temp_comment)

            if temp_ac == '人均':
                item['avg_consumption'] = '暂无人均消费'
            else:
                item['avg_consumption'] = temp_ac

            item['addr'] = temp_addr

            yield item
